{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "feature_list.csv      label_eventsV1.2.csv  raw_events.csv\r\n",
      "label_eventsV1.1.csv  label_eventsV1.csv    xxxx.csv\r\n"
     ]
    }
   ],
   "source": [
    "!ls /data/csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DCUBE_PATH=/data/jupyter_root/dcube_data/\r\n",
      "DCUBE_INPUT_PATH=$DCUBE_PATH\r\n",
      "DCUBE_OUTPUT_PATH=$DCUBE_PATH\r\n",
      "N=$1    # number of input files\r\n",
      "D='3' # dimensions of dcube\r\n",
      "rm -rf ${DCUBE_PATH}feature*/\r\n",
      "for ((i=1; i<=N; i +=5)) \r\n",
      "do\r\n",
      "    mkdir -p ${DCUBE_PATH}feature$(($i))\r\n",
      "    mkdir -p ${DCUBE_PATH}feature$(($i+1))\r\n",
      "    mkdir -p ${DCUBE_PATH}feature$(($i+2))\r\n",
      "    mkdir -p ${DCUBE_PATH}feature$(($i+3))\r\n",
      "    mkdir -p ${DCUBE_PATH}feature$(($i+4))  \r\n",
      "    # read the first line of the input file, and count ','---> dimensions of dcube\r\n",
      "    D1=$(head -n 1 ${DCUBE_PATH}/feature$(($i)).txt | grep -o ',' | wc -l)\r\n",
      "    D2=$(head -n 1 ${DCUBE_PATH}/feature$(($i+1)).txt | grep -o ',' | wc -l)\r\n",
      "    D3=$(head -n 1 ${DCUBE_PATH}/feature$(($i+2)).txt | grep -o ',' | wc -l)\r\n",
      "    D4=$(head -n 1 ${DCUBE_PATH}/feature$(($i+3)).txt | grep -o ',' | wc -l)\r\n",
      "    D5=$(head -n 1 ${DCUBE_PATH}/feature$(($i+4)).txt | grep -o ',' | wc -l)\r\n",
      "\r\n",
      "    java -cp ./DCube-1.0.jar dcube.Proposed ${DCUBE_PATH}/feature$(($i)).txt   ${DCUBE_PATH}/feature$(($i)) ${D1} geo density  5 &\\\r\n",
      "    java -cp ./DCube-1.0.jar dcube.Proposed ${DCUBE_PATH}/feature$(($i+1)).txt ${DCUBE_PATH}/feature$(($i+1)) ${D2} geo density  5 &\\\r\n",
      "    java -cp ./DCube-1.0.jar dcube.Proposed ${DCUBE_PATH}/feature$(($i+2)).txt ${DCUBE_PATH}/feature$(($i+2)) ${D3} geo density  5 &\\\r\n",
      "    java -cp ./DCube-1.0.jar dcube.Proposed ${DCUBE_PATH}/feature$(($i+3)).txt ${DCUBE_PATH}/feature$(($i+3)) ${D4} geo density  5 &\\\r\n",
      "    java -cp ./DCube-1.0.jar dcube.Proposed ${DCUBE_PATH}/feature$(($i+4)).txt ${DCUBE_PATH}/feature$(($i+4)) ${D5} geo density  5 \r\n",
      "    wait\r\n",
      "done\r\n",
      "for((i=1;i<=N;i++))\r\n",
      "do\r\n",
      "    # combine blocks to 'blocks.txt'\r\n",
      "    cat ${DCUBE_PATH}/feature${i}/block_1.tuples   ${DCUBE_PATH}/feature${i}/block_2.tuples > ${DCUBE_PATH}/feature${i}/block12.txt\r\n",
      "    cat ${DCUBE_PATH}/feature${i}/block_1.tuples   ${DCUBE_PATH}/feature${i}/block_3.tuples > ${DCUBE_PATH}/feature${i}/block13.txt\r\n",
      "    cat ${DCUBE_PATH}/feature${i}/block_2.tuples   ${DCUBE_PATH}/feature${i}/block_3.tuples > ${DCUBE_PATH}/feature${i}/block23.txt\r\n",
      "    cat ${DCUBE_PATH}/feature${i}/block_1.tuples   ${DCUBE_PATH}/feature${i}/block_2.tuples ${DCUBE_PATH}/feature${i}/block_3.tuples > ${DCUBE_PATH}/feature${i}/block123.txt\r\n",
      "done\r\n",
      "\r\n",
      "echo \"Over,run $N times!\"\r\n",
      "\r\n"
     ]
    }
   ],
   "source": [
    "!cat /data/jupyter_root/dcube_data/run1-nB.sh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3058: DtypeWarning: Columns (10) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n",
      "/opt/anaconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3058: DtypeWarning: Columns (11) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import pandas as pd\n",
    "import datetime as dt\n",
    "dataset = pd.read_csv(open('/data/csv/label_eventsV1.1.csv','r',encoding = 'gb18030'))\n",
    "dataset.to_csv('/data/csv/label_eventsV1.2.csv',index=True,encoding = 'gb18030')\n",
    "dataset = pd.read_csv(open('/data/csv/label_eventsV1.2.csv','r',encoding = 'gb18030'))\n",
    "\n",
    "dataset['time_stamp']=pd.to_datetime(dataset['time_stamp'])#识别为时间格式\n",
    "dataset['time_stamp_day']=dataset['time_stamp'].dt.strftime('%d')#只保留天格式\n",
    "dataset['time_stamp_hour']=dataset['time_stamp'].dt.strftime('%d%H')#只保留到小时格式\n",
    "\n",
    "dic_city={}\n",
    "i=0\n",
    "for city in set(dataset['ip_city']):\n",
    "    dic_city[city]=i\n",
    "    i+=1\n",
    "dataset['ip_city']=dataset['ip_city'].map(dic_city)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def time_to(time,interval):\n",
    "    start=dt.datetime(2017,10,1,0,0,0)\n",
    "    while(start<dt.datetime(2017,10,31,23,59,59)):\n",
    "#         start_str=start.strftime('%d %H')\n",
    "        if time>=start and time<start+dt.timedelta(hours=interval):\n",
    "            return start.strftime('%d%H')\n",
    "        else:\n",
    "            start+=dt.timedelta(hours=interval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3hour: 49.26251482963562\n",
      "6hour: 24.932595014572144\n"
     ]
    }
   ],
   "source": [
    "time1=time.time()\n",
    "dataset['time_stamp_3hour']=dataset['time_stamp'].apply(lambda x:time_to(x,3))\n",
    "time2=time.time()\n",
    "dataset['time_stamp_6hour']=dataset['time_stamp'].apply(lambda x:time_to(x,6))\n",
    "time3=time.time()\n",
    "print('3hour:',time2-time1)\n",
    "print('6hour:',time3-time2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>ip_1</th>\n",
       "      <th>ip_2</th>\n",
       "      <th>ip_3</th>\n",
       "      <th>ip_4</th>\n",
       "      <th>ip</th>\n",
       "      <th>ip_city</th>\n",
       "      <th>email_prefix</th>\n",
       "      <th>email_provider</th>\n",
       "      <th>event_type</th>\n",
       "      <th>...</th>\n",
       "      <th>resource_type</th>\n",
       "      <th>resource_category</th>\n",
       "      <th>label</th>\n",
       "      <th>time_stamp_day</th>\n",
       "      <th>time_stamp_hour</th>\n",
       "      <th>time_stamp_3hour</th>\n",
       "      <th>time_stamp_6hour</th>\n",
       "      <th>ip_12</th>\n",
       "      <th>ip_123</th>\n",
       "      <th>ip_1234</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>2515</td>\n",
       "      <td>2515</td>\n",
       "      <td>2512</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>61.0</td>\n",
       "      <td>162.0</td>\n",
       "      <td>94.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>61.162.94.25</td>\n",
       "      <td>51</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>1111</td>\n",
       "      <td>1109</td>\n",
       "      <td>1106</td>\n",
       "      <td>61162.0</td>\n",
       "      <td>61162094.0</td>\n",
       "      <td>-1.113433e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>112.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>127.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>112.64.127.42</td>\n",
       "      <td>33</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>3011</td>\n",
       "      <td>3009</td>\n",
       "      <td>3006</td>\n",
       "      <td>112064.0</td>\n",
       "      <td>112064127.0</td>\n",
       "      <td>-2.642086e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>60.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>206.0</td>\n",
       "      <td>63.0</td>\n",
       "      <td>60.16.206.63</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>2808</td>\n",
       "      <td>2806</td>\n",
       "      <td>2806</td>\n",
       "      <td>60016.0</td>\n",
       "      <td>60016206.0</td>\n",
       "      <td>-1.139749e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>61.0</td>\n",
       "      <td>153.0</td>\n",
       "      <td>148.0</td>\n",
       "      <td>192.0</td>\n",
       "      <td>61.153.148.192</td>\n",
       "      <td>67</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>07</td>\n",
       "      <td>0715</td>\n",
       "      <td>0715</td>\n",
       "      <td>0712</td>\n",
       "      <td>61153.0</td>\n",
       "      <td>61153148.0</td>\n",
       "      <td>-1.114008e+09</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0   ip_1   ip_2   ip_3   ip_4              ip  ip_city  \\\n",
       "0           0    NaN    NaN    NaN    NaN             NaN        0   \n",
       "1           1   61.0  162.0   94.0   25.0    61.162.94.25       51   \n",
       "2           2  112.0   64.0  127.0   42.0   112.64.127.42       33   \n",
       "3           3   60.0   16.0  206.0   63.0    60.16.206.63        2   \n",
       "4           4   61.0  153.0  148.0  192.0  61.153.148.192       67   \n",
       "\n",
       "   email_prefix  email_provider  event_type  ...  resource_type  \\\n",
       "0           NaN             NaN           0  ...            NaN   \n",
       "1           NaN             NaN           0  ...            NaN   \n",
       "2           NaN             NaN           0  ...            NaN   \n",
       "3           NaN             NaN           0  ...            NaN   \n",
       "4           NaN             NaN           0  ...            NaN   \n",
       "\n",
       "  resource_category label  time_stamp_day  time_stamp_hour  time_stamp_3hour  \\\n",
       "0               NaN     0              25             2515              2515   \n",
       "1               NaN     0              11             1111              1109   \n",
       "2               NaN     0              30             3011              3009   \n",
       "3               NaN     0              28             2808              2806   \n",
       "4               NaN     0              07             0715              0715   \n",
       "\n",
       "   time_stamp_6hour     ip_12       ip_123       ip_1234  \n",
       "0              2512       NaN          NaN           NaN  \n",
       "1              1106   61162.0   61162094.0 -1.113433e+09  \n",
       "2              3006  112064.0  112064127.0 -2.642086e+08  \n",
       "3              2806   60016.0   60016206.0 -1.139749e+09  \n",
       "4              0712   61153.0   61153148.0 -1.114008e+09  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>ip_1</th>\n",
       "      <th>ip_2</th>\n",
       "      <th>ip_3</th>\n",
       "      <th>ip_4</th>\n",
       "      <th>ip</th>\n",
       "      <th>ip_city</th>\n",
       "      <th>email_prefix</th>\n",
       "      <th>email_provider</th>\n",
       "      <th>event_type</th>\n",
       "      <th>...</th>\n",
       "      <th>resource_type</th>\n",
       "      <th>resource_category</th>\n",
       "      <th>label</th>\n",
       "      <th>time_stamp_day</th>\n",
       "      <th>time_stamp_hour</th>\n",
       "      <th>time_stamp_3hour</th>\n",
       "      <th>time_stamp_6hour</th>\n",
       "      <th>ip_12</th>\n",
       "      <th>ip_123</th>\n",
       "      <th>ip_1234</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>2515</td>\n",
       "      <td>2515</td>\n",
       "      <td>2512</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>61.0</td>\n",
       "      <td>162.0</td>\n",
       "      <td>94.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>61.162.94.25</td>\n",
       "      <td>51</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>1111</td>\n",
       "      <td>1109</td>\n",
       "      <td>1106</td>\n",
       "      <td>61162.0</td>\n",
       "      <td>61162094.0</td>\n",
       "      <td>517025548.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>112.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>127.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>112.64.127.42</td>\n",
       "      <td>33</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>3011</td>\n",
       "      <td>3009</td>\n",
       "      <td>3006</td>\n",
       "      <td>112064.0</td>\n",
       "      <td>112064127.0</td>\n",
       "      <td>941637525.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>60.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>206.0</td>\n",
       "      <td>63.0</td>\n",
       "      <td>60.16.206.63</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>2808</td>\n",
       "      <td>2806</td>\n",
       "      <td>2806</td>\n",
       "      <td>60016.0</td>\n",
       "      <td>60016206.0</td>\n",
       "      <td>503867167.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>61.0</td>\n",
       "      <td>153.0</td>\n",
       "      <td>148.0</td>\n",
       "      <td>192.0</td>\n",
       "      <td>61.153.148.192</td>\n",
       "      <td>67</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>07</td>\n",
       "      <td>0715</td>\n",
       "      <td>0715</td>\n",
       "      <td>0712</td>\n",
       "      <td>61153.0</td>\n",
       "      <td>61153148.0</td>\n",
       "      <td>516737632.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0   ip_1   ip_2   ip_3   ip_4              ip  ip_city  \\\n",
       "0           0    NaN    NaN    NaN    NaN             NaN        0   \n",
       "1           1   61.0  162.0   94.0   25.0    61.162.94.25       51   \n",
       "2           2  112.0   64.0  127.0   42.0   112.64.127.42       33   \n",
       "3           3   60.0   16.0  206.0   63.0    60.16.206.63        2   \n",
       "4           4   61.0  153.0  148.0  192.0  61.153.148.192       67   \n",
       "\n",
       "   email_prefix  email_provider  event_type  ...  resource_type  \\\n",
       "0           NaN             NaN           0  ...            NaN   \n",
       "1           NaN             NaN           0  ...            NaN   \n",
       "2           NaN             NaN           0  ...            NaN   \n",
       "3           NaN             NaN           0  ...            NaN   \n",
       "4           NaN             NaN           0  ...            NaN   \n",
       "\n",
       "  resource_category label  time_stamp_day  time_stamp_hour  time_stamp_3hour  \\\n",
       "0               NaN     0              25             2515              2515   \n",
       "1               NaN     0              11             1111              1109   \n",
       "2               NaN     0              30             3011              3009   \n",
       "3               NaN     0              28             2808              2806   \n",
       "4               NaN     0              07             0715              0715   \n",
       "\n",
       "   time_stamp_6hour     ip_12       ip_123      ip_1234  \n",
       "0              2512       NaN          NaN          NaN  \n",
       "1              1106   61162.0   61162094.0  517025548.0  \n",
       "2              3006  112064.0  112064127.0  941637525.0  \n",
       "3              2806   60016.0   60016206.0  503867167.0  \n",
       "4              0712   61153.0   61153148.0  516737632.0  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def float_to_str(x):\n",
    "    if str(x)=='nan':\n",
    "        return 'nan'\n",
    "    else:\n",
    "        xs=str(int(x))\n",
    "        while len(xs)<3:\n",
    "            xs='0'+xs\n",
    "        return xs\n",
    "def str_to_float(x):\n",
    "    if x[0]=='n':\n",
    "        return np.nan\n",
    "    else:\n",
    "        return float(x)\n",
    "dataset['ip_12']=(dataset['ip_1'].apply(float_to_str)+dataset['ip_2'].apply(float_to_str)).apply(str_to_float)\n",
    "dataset['ip_123']=(dataset['ip_1'].apply(float_to_str)+dataset['ip_2'].apply(float_to_str)+dataset['ip_3'].apply(float_to_str)).apply(str_to_float)\n",
    "\n",
    "from IPy import IP\n",
    "def ip_to_int32(x): \n",
    "    if type(x)==str:\n",
    "        return float(IP(x).int()//2)\n",
    "dataset['ip_1234']=dataset['ip'].apply(ip_to_int32)\n",
    "dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 69521 entries, 0 to 69520\n",
      "Data columns (total 30 columns):\n",
      "Unnamed: 0           69521 non-null int64\n",
      "ip_1                 30676 non-null float64\n",
      "ip_2                 30676 non-null float64\n",
      "ip_3                 30676 non-null float64\n",
      "ip_4                 30676 non-null float64\n",
      "ip                   30676 non-null object\n",
      "ip_city              69521 non-null int64\n",
      "email_prefix         35 non-null float64\n",
      "email_provider       35 non-null float64\n",
      "event_type           69521 non-null int64\n",
      "mobile_prefix_3      243 non-null float64\n",
      "mobile_city          243 non-null object\n",
      "time_stamp           69521 non-null datetime64[ns]\n",
      "user_name            69521 non-null int64\n",
      "user_agent           69521 non-null int64\n",
      "os_version           69521 non-null int64\n",
      "resource_owner       36631 non-null float64\n",
      "register_type        333 non-null float64\n",
      "category             3533 non-null float64\n",
      "status               3533 non-null float64\n",
      "resource_type        36961 non-null float64\n",
      "resource_category    36961 non-null float64\n",
      "label                69521 non-null int64\n",
      "time_stamp_day       69521 non-null object\n",
      "time_stamp_hour      69521 non-null object\n",
      "time_stamp_3hour     69521 non-null object\n",
      "time_stamp_6hour     69521 non-null object\n",
      "ip_12                30676 non-null float64\n",
      "ip_123               30676 non-null float64\n",
      "ip_1234              30676 non-null float64\n",
      "dtypes: datetime64[ns](1), float64(16), int64(7), object(6)\n",
      "memory usage: 15.9+ MB\n"
     ]
    }
   ],
   "source": [
    "dataset.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(69521, 30)\n",
      "(65325, 30)\n"
     ]
    }
   ],
   "source": [
    "print(dataset.shape)\n",
    "dataset_12=dataset[(dataset.event_type==1) | (dataset.event_type==2)]\n",
    "print(dataset_12.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['user_name']\n",
      "user_agent\n",
      "nunique\n",
      "['user_name', 'time_stamp_day']\n",
      "user_agent\n",
      "nunique\n",
      "['user_name', 'time_stamp_hour']\n",
      "user_agent\n",
      "nunique\n",
      "['user_name', 'time_stamp_3hour']\n",
      "user_agent\n",
      "nunique\n",
      "['user_name', 'time_stamp_6hour']\n",
      "user_agent\n",
      "nunique\n",
      "['user_name']\n",
      "os_version\n",
      "nunique\n",
      "['user_name', 'time_stamp_day']\n",
      "os_version\n",
      "nunique\n",
      "['user_name', 'time_stamp_hour']\n",
      "os_version\n",
      "nunique\n",
      "['user_name', 'time_stamp_3hour']\n",
      "os_version\n",
      "nunique\n",
      "['user_name', 'time_stamp_6hour']\n",
      "os_version\n",
      "nunique\n",
      "['user_name']\n",
      "ip_1\n",
      "nunique\n",
      "['user_name', 'time_stamp_day']\n",
      "ip_1\n",
      "nunique\n",
      "['user_name', 'time_stamp_hour']\n",
      "ip_1\n",
      "nunique\n",
      "['user_name', 'time_stamp_3hour']\n",
      "ip_1\n",
      "nunique\n",
      "['user_name', 'time_stamp_6hour']\n",
      "ip_1\n",
      "nunique\n",
      "['user_name']\n",
      "ip_12\n",
      "nunique\n",
      "['user_name', 'time_stamp_day']\n",
      "ip_12\n",
      "nunique\n",
      "['user_name', 'time_stamp_hour']\n",
      "ip_12\n",
      "nunique\n",
      "['user_name', 'time_stamp_3hour']\n",
      "ip_12\n",
      "nunique\n",
      "['user_name', 'time_stamp_6hour']\n",
      "ip_12\n",
      "nunique\n",
      "['user_name']\n",
      "ip_123\n",
      "nunique\n",
      "['user_name', 'time_stamp_day']\n",
      "ip_123\n",
      "nunique\n",
      "['user_name', 'time_stamp_hour']\n",
      "ip_123\n",
      "nunique\n",
      "['user_name', 'time_stamp_3hour']\n",
      "ip_123\n",
      "nunique\n",
      "['user_name', 'time_stamp_6hour']\n",
      "ip_123\n",
      "nunique\n",
      "['user_name']\n",
      "ip_1234\n",
      "nunique\n",
      "['user_name', 'time_stamp_day']\n",
      "ip_1234\n",
      "nunique\n",
      "['user_name', 'time_stamp_hour']\n",
      "ip_1234\n",
      "nunique\n",
      "['user_name', 'time_stamp_3hour']\n",
      "ip_1234\n",
      "nunique\n",
      "['user_name', 'time_stamp_6hour']\n",
      "ip_1234\n",
      "nunique\n",
      "['user_name']\n",
      "ip_city\n",
      "nunique\n",
      "['user_name', 'time_stamp_day']\n",
      "ip_city\n",
      "nunique\n",
      "['user_name', 'time_stamp_hour']\n",
      "ip_city\n",
      "nunique\n",
      "['user_name', 'time_stamp_3hour']\n",
      "ip_city\n",
      "nunique\n",
      "['user_name', 'time_stamp_6hour']\n",
      "ip_city\n",
      "nunique\n",
      "['user_name']\n",
      "resource_owner\n",
      "nunique\n",
      "['user_name', 'time_stamp_day']\n",
      "resource_owner\n",
      "nunique\n",
      "['user_name', 'time_stamp_hour']\n",
      "resource_owner\n",
      "nunique\n",
      "['user_name', 'time_stamp_3hour']\n",
      "resource_owner\n",
      "nunique\n",
      "['user_name', 'time_stamp_6hour']\n",
      "resource_owner\n",
      "nunique\n",
      "['user_name']\n",
      "resource_type\n",
      "nunique\n",
      "['user_name', 'time_stamp_day']\n",
      "resource_type\n",
      "nunique\n",
      "['user_name', 'time_stamp_hour']\n",
      "resource_type\n",
      "nunique\n",
      "['user_name', 'time_stamp_3hour']\n",
      "resource_type\n",
      "nunique\n",
      "['user_name', 'time_stamp_6hour']\n",
      "resource_type\n",
      "nunique\n",
      "['user_name']\n",
      "resource_category\n",
      "nunique\n",
      "['user_name', 'time_stamp_day']\n",
      "resource_category\n",
      "nunique\n",
      "['user_name', 'time_stamp_hour']\n",
      "resource_category\n",
      "nunique\n",
      "['user_name', 'time_stamp_3hour']\n",
      "resource_category\n",
      "nunique\n",
      "['user_name', 'time_stamp_6hour']\n",
      "resource_category\n",
      "nunique\n",
      "['user_name', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'time_stamp_day']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'time_stamp_hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'time_stamp_3hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'time_stamp_6hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'user_agent', 'time_stamp_day']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'user_agent', 'time_stamp_hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'user_agent', 'time_stamp_3hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'user_agent', 'time_stamp_6hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'os_version', 'time_stamp_day']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'os_version', 'time_stamp_hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'os_version', 'time_stamp_3hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'os_version', 'time_stamp_6hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_1', 'time_stamp_day']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_1', 'time_stamp_hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_1', 'time_stamp_3hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_1', 'time_stamp_6hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_12', 'time_stamp_day']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_12', 'time_stamp_hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_12', 'time_stamp_3hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_12', 'time_stamp_6hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_123', 'time_stamp_day']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_123', 'time_stamp_hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_123', 'time_stamp_3hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_123', 'time_stamp_6hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_1234', 'time_stamp_day']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_1234', 'time_stamp_hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_1234', 'time_stamp_3hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_1234', 'time_stamp_6hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_city', 'time_stamp_day']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_city', 'time_stamp_hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_city', 'time_stamp_3hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_city', 'time_stamp_6hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_owner', 'time_stamp_day']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_owner', 'time_stamp_hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_owner', 'time_stamp_3hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_owner', 'time_stamp_6hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_type', 'time_stamp_day']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_type', 'time_stamp_hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_type', 'time_stamp_3hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_type', 'time_stamp_6hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_category', 'time_stamp_day']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_category', 'time_stamp_hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_category', 'time_stamp_3hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_category', 'time_stamp_6hour']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'user_agent', 'time_stamp_day', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'user_agent', 'time_stamp_hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'user_agent', 'time_stamp_3hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'user_agent', 'time_stamp_6hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'os_version', 'time_stamp_day', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'os_version', 'time_stamp_hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'os_version', 'time_stamp_3hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'os_version', 'time_stamp_6hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_1', 'time_stamp_day', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_1', 'time_stamp_hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_1', 'time_stamp_3hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_1', 'time_stamp_6hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_12', 'time_stamp_day', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_12', 'time_stamp_hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_12', 'time_stamp_3hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_12', 'time_stamp_6hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_123', 'time_stamp_day', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_123', 'time_stamp_hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_123', 'time_stamp_3hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_123', 'time_stamp_6hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_1234', 'time_stamp_day', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_1234', 'time_stamp_hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_1234', 'time_stamp_3hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_1234', 'time_stamp_6hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_city', 'time_stamp_day', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_city', 'time_stamp_hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_city', 'time_stamp_3hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'ip_city', 'time_stamp_6hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_owner', 'time_stamp_day', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_owner', 'time_stamp_hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_owner', 'time_stamp_3hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_owner', 'time_stamp_6hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_type', 'time_stamp_day', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_type', 'time_stamp_hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_type', 'time_stamp_3hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_type', 'time_stamp_6hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_category', 'time_stamp_day', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_category', 'time_stamp_hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_category', 'time_stamp_3hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n",
      "['user_name', 'resource_category', 'time_stamp_6hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n"
     ]
    }
   ],
   "source": [
    "device=['user_agent','os_version','ip_1','ip_12','ip_123','ip_1234','ip_city','resource_owner','resource_type','resource_category']\n",
    "grouplist=[]\n",
    "    \n",
    "'''\n",
    "    #[设备][用户]   用户个数的设备聚集（同一个设备对应多少不同用户）\n",
    "    grouplist.append([[i],'user_name','nunique'])\n",
    "    \n",
    "    #[设备，时间][用户] 用户个数的设备聚集、时间聚集（同一时间同一设备有多少不同用户）\n",
    "    grouplist.append([[i,'time_stamp'],'user_name','nunique'])\n",
    "'''\n",
    "\n",
    "for i in device:\n",
    "    #[用户][设备]   设备个数的用户聚集（同一个用户对应多少不同设备）\n",
    "    grouplist.append([['user_name'],i,'nunique'])\n",
    "    \n",
    "    # [用户，时间][设备] 用户个数的设备聚集、时间聚集（同一时间同一设备有多少不同用户）\n",
    "    grouplist.append([['user_name','time_stamp_day'],i,'nunique'])\n",
    "    grouplist.append([['user_name','time_stamp_hour'],i,'nunique'])\n",
    "    grouplist.append([['user_name','time_stamp_3hour'],i,'nunique'])\n",
    "    grouplist.append([['user_name','time_stamp_6hour'],i,'nunique'])\n",
    "\n",
    "    \n",
    "#[用户,某个事件类型][事件]  事件的事件类型、用户聚集（同一事件类型，同一用户进行多少次）\n",
    "grouplist.append([['user_name','event_type'],'Unnamed: 0','count'])\n",
    "\n",
    "#[用户，时间][总事件数]（时间为周，天）--看用户进行登陆等事件是否具有明显的时间集中性                                                          \n",
    "grouplist.append([['user_name','time_stamp_day'],'Unnamed: 0','count'])\n",
    "grouplist.append([['user_name','time_stamp_hour'],'Unnamed: 0','count'])\n",
    "grouplist.append([['user_name','time_stamp_3hour'],'Unnamed: 0','count'])\n",
    "grouplist.append([['user_name','time_stamp_6hour'],'Unnamed: 0','count'])\n",
    "\n",
    "#[用户,设备，时间][总事件]\n",
    "for i in device:\n",
    "    grouplist.append([['user_name',i,'time_stamp_day'],'Unnamed: 0','count'])\n",
    "    grouplist.append([['user_name',i,'time_stamp_hour'],'Unnamed: 0','count'])\n",
    "    grouplist.append([['user_name',i,'time_stamp_3hour'],'Unnamed: 0','count'])\n",
    "    grouplist.append([['user_name',i,'time_stamp_6hour'],'Unnamed: 0','count'])\n",
    "    \n",
    "#[用户,设备，时间,事件类型][总事件]\n",
    "for i in device:\n",
    "    grouplist.append([['user_name',i,'time_stamp_day','event_type'],'Unnamed: 0','count'])\n",
    "    grouplist.append([['user_name',i,'time_stamp_hour','event_type'],'Unnamed: 0','count'])\n",
    "    grouplist.append([['user_name',i,'time_stamp_3hour','event_type'],'Unnamed: 0','count'])\n",
    "    grouplist.append([['user_name',i,'time_stamp_6hour','event_type'],'Unnamed: 0','count'])\n",
    "\n",
    "user_name_dic=[]\n",
    "i=1\n",
    "for x in grouplist:\n",
    "    print(x[0])\n",
    "    print(x[1])\n",
    "    print(x[2])\n",
    "    save_df=dataset_12.groupby(x[0],as_index=False).agg({x[1]:x[2]})\n",
    "    for c in range(save_df.shape[1]):\n",
    "        save_df.iloc[:,c]=save_df.iloc[:,c].apply(int)\n",
    "    user_name_dic.append(set(save_df['user_name']))\n",
    "    save_df.to_csv('/data/jupyter_root/dcube_data/feature'+str(i)+'.txt',sep=',',index=None,header=None)\n",
    "    i+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8-4-10.log\tfeature12.txt\tfeature30      feature50.txt  feature71\r\n",
      "8-4-174.log\tfeature13\tfeature30.txt  feature51      feature71.txt\r\n",
      "8-5-1.log\tfeature130.txt\tfeature31      feature51.txt  feature72\r\n",
      "8-5-2.log\tfeature131.txt\tfeature31.txt  feature52      feature72.txt\r\n",
      "clean.sh\tfeature132.txt\tfeature32      feature52.txt  feature73\r\n",
      "DCube-1.0.jar\tfeature133.txt\tfeature32.txt  feature53      feature73.txt\r\n",
      "DCube-2.0.jar\tfeature134.txt\tfeature33      feature53.txt  feature74.txt\r\n",
      "Density.txt\tfeature135.txt\tfeature33.txt  feature54      feature75.txt\r\n",
      "evaluate.txt\tfeature13.txt\tfeature34      feature54.txt  feature76.txt\r\n",
      "feature1\tfeature14\tfeature34.txt  feature55      feature77.txt\r\n",
      "feature10\tfeature14.txt\tfeature35      feature55.txt  feature78.txt\r\n",
      "feature100.txt\tfeature15\tfeature35.txt  feature56      feature79.txt\r\n",
      "feature101.txt\tfeature15.txt\tfeature36      feature56.txt  feature7.txt\r\n",
      "feature102.txt\tfeature16\tfeature36.txt  feature57      feature8\r\n",
      "feature103.txt\tfeature16.txt\tfeature37      feature57.txt  feature80.txt\r\n",
      "feature104.txt\tfeature17\tfeature37.txt  feature58      feature81.txt\r\n",
      "feature105.txt\tfeature17.txt\tfeature38      feature58.txt  feature82.txt\r\n",
      "feature106.txt\tfeature18\tfeature38.txt  feature59      feature83.txt\r\n",
      "feature107.txt\tfeature18.txt\tfeature39      feature59.txt  feature84.txt\r\n",
      "feature108.txt\tfeature19\tfeature39.txt  feature5.txt   feature85.txt\r\n",
      "feature109.txt\tfeature19.txt\tfeature3.txt   feature6       feature86.txt\r\n",
      "feature10.txt\tfeature1.txt\tfeature4       feature60      feature87.txt\r\n",
      "feature11\tfeature2\tfeature40      feature60.txt  feature88.txt\r\n",
      "feature110.txt\tfeature20\tfeature40.txt  feature61      feature89.txt\r\n",
      "feature111.txt\tfeature20.txt\tfeature41      feature61.txt  feature8.txt\r\n",
      "feature112.txt\tfeature21\tfeature41.txt  feature62      feature9\r\n",
      "feature113.txt\tfeature21.txt\tfeature42      feature62.txt  feature90.txt\r\n",
      "feature114.txt\tfeature22\tfeature42.txt  feature63      feature91.txt\r\n",
      "feature115.txt\tfeature22.txt\tfeature43      feature63.txt  feature92.txt\r\n",
      "feature116.txt\tfeature23\tfeature43.txt  feature64      feature93.txt\r\n",
      "feature117.txt\tfeature23.txt\tfeature44      feature64.txt  feature94.txt\r\n",
      "feature118.txt\tfeature24\tfeature44.txt  feature65      feature95.txt\r\n",
      "feature119.txt\tfeature24.txt\tfeature45      feature65.txt  feature96.txt\r\n",
      "feature11.txt\tfeature25\tfeature45.txt  feature66      feature97.txt\r\n",
      "feature12\tfeature25.txt\tfeature46      feature66.txt  feature98.txt\r\n",
      "feature120.txt\tfeature26\tfeature46.txt  feature67      feature99.txt\r\n",
      "feature121.txt\tfeature26.txt\tfeature47      feature67.txt  feature9.txt\r\n",
      "feature122.txt\tfeature27\tfeature47.txt  feature68      get-blocks.sh\r\n",
      "feature123.txt\tfeature27.txt\tfeature48      feature68.txt  README.txt\r\n",
      "feature124.txt\tfeature28\tfeature48.txt  feature69      run1-nB.sh\r\n",
      "feature125.txt\tfeature28.txt\tfeature49      feature69.txt  run1.sh\r\n",
      "feature126.txt\tfeature29\tfeature49.txt  feature6.txt   run2.sh\r\n",
      "feature127.txt\tfeature29.txt\tfeature4.txt   feature7       run3.sh\r\n",
      "feature128.txt\tfeature2.txt\tfeature5       feature70      run.sh\r\n",
      "feature129.txt\tfeature3\tfeature50      feature70.txt  run-single.sh\r\n"
     ]
    }
   ],
   "source": [
    "!ls /data/jupyter_root/dcube_data/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Feature_combination</th>\n",
       "      <th>Standard</th>\n",
       "      <th>Poly</th>\n",
       "      <th>user_name_dic</th>\n",
       "      <th>user_name_num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>feature1</th>\n",
       "      <td>[user_name]</td>\n",
       "      <td>user_agent</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature2</th>\n",
       "      <td>[user_name, time_stamp_day]</td>\n",
       "      <td>user_agent</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature3</th>\n",
       "      <td>[user_name, time_stamp_hour]</td>\n",
       "      <td>user_agent</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature4</th>\n",
       "      <td>[user_name, time_stamp_3hour]</td>\n",
       "      <td>user_agent</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature5</th>\n",
       "      <td>[user_name, time_stamp_6hour]</td>\n",
       "      <td>user_agent</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature6</th>\n",
       "      <td>[user_name]</td>\n",
       "      <td>os_version</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature7</th>\n",
       "      <td>[user_name, time_stamp_day]</td>\n",
       "      <td>os_version</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature8</th>\n",
       "      <td>[user_name, time_stamp_hour]</td>\n",
       "      <td>os_version</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature9</th>\n",
       "      <td>[user_name, time_stamp_3hour]</td>\n",
       "      <td>os_version</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature10</th>\n",
       "      <td>[user_name, time_stamp_6hour]</td>\n",
       "      <td>os_version</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature11</th>\n",
       "      <td>[user_name]</td>\n",
       "      <td>ip_1</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature12</th>\n",
       "      <td>[user_name, time_stamp_day]</td>\n",
       "      <td>ip_1</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature13</th>\n",
       "      <td>[user_name, time_stamp_hour]</td>\n",
       "      <td>ip_1</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature14</th>\n",
       "      <td>[user_name, time_stamp_3hour]</td>\n",
       "      <td>ip_1</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature15</th>\n",
       "      <td>[user_name, time_stamp_6hour]</td>\n",
       "      <td>ip_1</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature16</th>\n",
       "      <td>[user_name]</td>\n",
       "      <td>ip_12</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature17</th>\n",
       "      <td>[user_name, time_stamp_day]</td>\n",
       "      <td>ip_12</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature18</th>\n",
       "      <td>[user_name, time_stamp_hour]</td>\n",
       "      <td>ip_12</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature19</th>\n",
       "      <td>[user_name, time_stamp_3hour]</td>\n",
       "      <td>ip_12</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature20</th>\n",
       "      <td>[user_name, time_stamp_6hour]</td>\n",
       "      <td>ip_12</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature21</th>\n",
       "      <td>[user_name]</td>\n",
       "      <td>ip_123</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature22</th>\n",
       "      <td>[user_name, time_stamp_day]</td>\n",
       "      <td>ip_123</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature23</th>\n",
       "      <td>[user_name, time_stamp_hour]</td>\n",
       "      <td>ip_123</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature24</th>\n",
       "      <td>[user_name, time_stamp_3hour]</td>\n",
       "      <td>ip_123</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature25</th>\n",
       "      <td>[user_name, time_stamp_6hour]</td>\n",
       "      <td>ip_123</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature26</th>\n",
       "      <td>[user_name]</td>\n",
       "      <td>ip_1234</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature27</th>\n",
       "      <td>[user_name, time_stamp_day]</td>\n",
       "      <td>ip_1234</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature28</th>\n",
       "      <td>[user_name, time_stamp_hour]</td>\n",
       "      <td>ip_1234</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature29</th>\n",
       "      <td>[user_name, time_stamp_3hour]</td>\n",
       "      <td>ip_1234</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature30</th>\n",
       "      <td>[user_name, time_stamp_6hour]</td>\n",
       "      <td>ip_1234</td>\n",
       "      <td>nunique</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature106</th>\n",
       "      <td>[user_name, ip_1, time_stamp_3hour, event_type]</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{458762, 1310734, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature107</th>\n",
       "      <td>[user_name, ip_1, time_stamp_6hour, event_type]</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{458762, 1310734, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature108</th>\n",
       "      <td>[user_name, ip_12, time_stamp_day, event_type]</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{458762, 1310734, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature109</th>\n",
       "      <td>[user_name, ip_12, time_stamp_hour, event_type]</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{458762, 1310734, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature110</th>\n",
       "      <td>[user_name, ip_12, time_stamp_3hour, event_type]</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{458762, 1310734, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature111</th>\n",
       "      <td>[user_name, ip_12, time_stamp_6hour, event_type]</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{458762, 1310734, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature112</th>\n",
       "      <td>[user_name, ip_123, time_stamp_day, event_type]</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{458762, 1310734, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature113</th>\n",
       "      <td>[user_name, ip_123, time_stamp_hour, event_type]</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{458762, 1310734, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature114</th>\n",
       "      <td>[user_name, ip_123, time_stamp_3hour, event_type]</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{458762, 1310734, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature115</th>\n",
       "      <td>[user_name, ip_123, time_stamp_6hour, event_type]</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{458762, 1310734, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature116</th>\n",
       "      <td>[user_name, ip_1234, time_stamp_day, event_type]</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{458762, 1310734, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature117</th>\n",
       "      <td>[user_name, ip_1234, time_stamp_hour, event_type]</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{458762, 1310734, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature118</th>\n",
       "      <td>[user_name, ip_1234, time_stamp_3hour, event_t...</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{458762, 1310734, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature119</th>\n",
       "      <td>[user_name, ip_1234, time_stamp_6hour, event_t...</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{458762, 1310734, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature120</th>\n",
       "      <td>[user_name, ip_city, time_stamp_day, event_type]</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature121</th>\n",
       "      <td>[user_name, ip_city, time_stamp_hour, event_type]</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature122</th>\n",
       "      <td>[user_name, ip_city, time_stamp_3hour, event_t...</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature123</th>\n",
       "      <td>[user_name, ip_city, time_stamp_6hour, event_t...</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{1343494, 458762, 1310734, 1343510, 1540121, 1...</td>\n",
       "      <td>9409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature124</th>\n",
       "      <td>[user_name, resource_owner, time_stamp_day, ev...</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{1343494, 458762, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature125</th>\n",
       "      <td>[user_name, resource_owner, time_stamp_hour, e...</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{1343494, 458762, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature126</th>\n",
       "      <td>[user_name, resource_owner, time_stamp_3hour, ...</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{1343494, 458762, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature127</th>\n",
       "      <td>[user_name, resource_owner, time_stamp_6hour, ...</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{1343494, 458762, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature128</th>\n",
       "      <td>[user_name, resource_type, time_stamp_day, eve...</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{1343494, 458762, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature129</th>\n",
       "      <td>[user_name, resource_type, time_stamp_hour, ev...</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{1343494, 458762, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature130</th>\n",
       "      <td>[user_name, resource_type, time_stamp_3hour, e...</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{1343494, 458762, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature131</th>\n",
       "      <td>[user_name, resource_type, time_stamp_6hour, e...</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{1343494, 458762, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature132</th>\n",
       "      <td>[user_name, resource_category, time_stamp_day,...</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{1343494, 458762, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature133</th>\n",
       "      <td>[user_name, resource_category, time_stamp_hour...</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{1343494, 458762, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature134</th>\n",
       "      <td>[user_name, resource_category, time_stamp_3hou...</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{1343494, 458762, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature135</th>\n",
       "      <td>[user_name, resource_category, time_stamp_6hou...</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{1343494, 458762, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7408</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>135 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          Feature_combination    Standard  \\\n",
       "feature1                                          [user_name]  user_agent   \n",
       "feature2                          [user_name, time_stamp_day]  user_agent   \n",
       "feature3                         [user_name, time_stamp_hour]  user_agent   \n",
       "feature4                        [user_name, time_stamp_3hour]  user_agent   \n",
       "feature5                        [user_name, time_stamp_6hour]  user_agent   \n",
       "...                                                       ...         ...   \n",
       "feature131  [user_name, resource_type, time_stamp_6hour, e...  Unnamed: 0   \n",
       "feature132  [user_name, resource_category, time_stamp_day,...  Unnamed: 0   \n",
       "feature133  [user_name, resource_category, time_stamp_hour...  Unnamed: 0   \n",
       "feature134  [user_name, resource_category, time_stamp_3hou...  Unnamed: 0   \n",
       "feature135  [user_name, resource_category, time_stamp_6hou...  Unnamed: 0   \n",
       "\n",
       "               Poly                                      user_name_dic  \\\n",
       "feature1    nunique  {1343494, 458762, 1310734, 1343510, 1540121, 1...   \n",
       "feature2    nunique  {1343494, 458762, 1310734, 1343510, 1540121, 1...   \n",
       "feature3    nunique  {1343494, 458762, 1310734, 1343510, 1540121, 1...   \n",
       "feature4    nunique  {1343494, 458762, 1310734, 1343510, 1540121, 1...   \n",
       "feature5    nunique  {1343494, 458762, 1310734, 1343510, 1540121, 1...   \n",
       "...             ...                                                ...   \n",
       "feature131    count  {1343494, 458762, 1343510, 1540121, 1671194, 1...   \n",
       "feature132    count  {1343494, 458762, 1343510, 1540121, 1671194, 1...   \n",
       "feature133    count  {1343494, 458762, 1343510, 1540121, 1671194, 1...   \n",
       "feature134    count  {1343494, 458762, 1343510, 1540121, 1671194, 1...   \n",
       "feature135    count  {1343494, 458762, 1343510, 1540121, 1671194, 1...   \n",
       "\n",
       "            user_name_num  \n",
       "feature1             9409  \n",
       "feature2             9409  \n",
       "feature3             9409  \n",
       "feature4             9409  \n",
       "feature5             9409  \n",
       "...                   ...  \n",
       "feature131           7408  \n",
       "feature132           7408  \n",
       "feature133           7408  \n",
       "feature134           7408  \n",
       "feature135           7408  \n",
       "\n",
       "[135 rows x 5 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Feature_combination=[]\n",
    "Standard=[]\n",
    "Poly=[]\n",
    "for group in grouplist:\n",
    "    Feature_combination.append(group[0])\n",
    "    Standard.append(group[1])\n",
    "    Poly.append(group[2])\n",
    "lis_name=[]\n",
    "for i in range(1,len(grouplist)+1):\n",
    "    lis_name.append('feature'+str(i))\n",
    "feature_list=pd.DataFrame({'Feature_combination':Feature_combination,'Standard':Standard,'Poly':Poly,'user_name_dic':user_name_dic},index=lis_name)\n",
    "feature_list['user_name_num']=feature_list['user_name_dic'].apply(len)\n",
    "feature_list.to_csv('/data/csv/feature_list.csv')\n",
    "feature_list_mini=pd.DataFrame({'Feature_combination':Feature_combination,'Standard':Standard,'Poly':Poly},index=lis_name)\n",
    "feature_list_mini['user_name_num']=feature_list['user_name_num']\n",
    "feature_list_mini.to_csv('/data/jupyter_root/feature_list.csv')\n",
    "feature_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_list.to_csv('/data/jupyter_root/GSD/feature_list.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Feature_combination</th>\n",
       "      <th>Standard</th>\n",
       "      <th>Poly</th>\n",
       "      <th>user_name_dic</th>\n",
       "      <th>user_name_num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>feature83</th>\n",
       "      <td>[user_name, ip_1234, time_stamp_day]</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{458762, 1310734, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature86</th>\n",
       "      <td>[user_name, ip_1234, time_stamp_6hour]</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>count</td>\n",
       "      <td>{458762, 1310734, 1343510, 1540121, 1671194, 1...</td>\n",
       "      <td>7106</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              Feature_combination    Standard   Poly  \\\n",
       "feature83    [user_name, ip_1234, time_stamp_day]  Unnamed: 0  count   \n",
       "feature86  [user_name, ip_1234, time_stamp_6hour]  Unnamed: 0  count   \n",
       "\n",
       "                                               user_name_dic  user_name_num  \n",
       "feature83  {458762, 1310734, 1343510, 1540121, 1671194, 1...           7106  \n",
       "feature86  {458762, 1310734, 1343510, 1540121, 1671194, 1...           7106  "
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_list.loc[['feature83','feature86']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['user_name', 'ip_1234', 'time_stamp_hour', 'event_type']\n",
      "Unnamed: 0\n",
      "count\n"
     ]
    }
   ],
   "source": [
    "    x=grouplist[135]\n",
    "    print(x[0])\n",
    "    print(x[1])\n",
    "    print(x[2])\n",
    "    save_df=dataset_12.groupby(x[0],as_index=False).agg({x[1]:x[2]})\n",
    "    for c in range(save_df.shape[1]):\n",
    "        save_df.iloc[:,c]=save_df.iloc[:,c].apply(int)\n",
    "    save_df.to_csv('/data/jupyter_root/GSD/test1.txt',sep=',',index=None,header=None)\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
